#!/usr/local/bin/python
#-*- encoding:utf-8 -*-
import numpy as np;
import scipy.sparse as sp;
import time
import threading
import os
from multiprocessing import Process
import utils
import ConfigParser

config = ConfigParser.ConfigParser();
config.read('config');

n = 822790

#Use lil_matrix to represent;
def getRow(s, d):
    row  = [];
    li = s.strip().split(' ');
    for l in li:
        pair = l.strip().split(':');
        index = int(pair[0]);
        value = float(pair[1]);
        row.append((index-1,value))
    row.sort();
    return row;

def getRow2(s, d):
    row  = [];
    li = s.strip().split(' ');
    if len(s.strip())== 0:
        return row;
    for l in li:
        pair = l.strip().split(':');
        index = int(pair[0]);
        value = float(pair[1]);
        row.append((index-1,value))
    row.sort();
    return row;

#Only for lil_matrix
def normlize(S):
    C = S.mean(axis=0).getA1();
    print C;
    for i in range(S.shape[0]):
        for k in range(len(S.data[i])):
            S.data[i][k] -= C[S.rows[i][k]];

def saveTemp():
    utils.save_sparse_csc(config.get('rmls', 'WFile'), W);
    utils.save_sparse_csc(config.get('rmls', 'NFile'), N);

#compute M and N batch
def batchPreprocessing(featureDataFile):
    K = 20;
    featureData = open(featureDataFile, 'r');
    ni = 0;
    line = featureData.readline();
    if not line.startswith('*'):
        print 'No Dimension Provided';
        return;
    global dx, dy, W ,N;
    firstLi = line.split('\t');
    dx = int(firstLi[1]);
    dy = int(firstLi[2]);
    print 'dimension:', dx, dy;

    linec = 0;
    global W;
    global N;
    W = sp.csc_matrix((dy, dx), dtype= np.float64);
    N = sp.csc_matrix((dx, dy), dtype= np.float64);
    D =  sp.lil_matrix((n, dy), dtype = np.float64);
    P = sp.dia_matrix((n,n), dtype = np.float64);
    Q = sp.lil_matrix((n, dx), dtype = np.float64);
    dig = np.zeros(n, dtype=np.float64);
    c = 0;

    while(True):
        line = featureData.readline();
        c += 1;
        print 'line' + str(c);
        if not line:
            break;
        elif line.startswith('#'):
            li = line.split('\t');
            i = li[1];
            ni += 1;
            niy = int(li[2])
            
            for j in range(niy):
                    pair = featureData.readline();
                    instance = pair.split('\t');
                    res = float(instance[0]);
                    if instance[1].strip() == '' or instance[2].strip() == 'None':
                        continue;
                    xi = getRow2(instance[1].strip(), dx);
                    yij = getRow2(instance[2].strip(), dy);
                
                    a = zip(*xi);
                
                    if len(xi) == 0:
                        continue;
                    if len(yij) == 0:
                        continue;
                    Q.rows[linec], Q.data[linec] = map(list, zip(*xi));
                    D.rows[linec], D.data[linec] = map(list, zip(*yij));
                    dig[linec] = res/float(niy);
                    linec += 1;

    dig  = dig /float(ni);
    P.setdiag(dig);
    print 'XXXXX'
    print P;
    #normlize(D);
    #normlize(Q);

    W = D.tocsr().transpose().dot(P).dot(Q.tocsr()).tocsc();
    N = Q.tocsr().transpose().dot(P).dot(D.tocsr()).tocsc();
   # print W;
    saveTemp();
    featureData.close();

def FroNorm(M):
    sum = 0.0;
    for a in M.data:
        sum += a*a
    return sum;
if __name__ == '__main__':
    featureDataFile = config.get('rmls', 'train.set');
    batchPreprocessing(featureDataFile);
    
